const cheerio = require('cheerio')
const axios = require('axios')
const iconv = require('iconv-lite')
const fs = require('fs')
const path = require('path')
const request = require('request')
// 抓取网站的链接
function urlByPage(page){
    return `http://md.itlun.cn/a/new/list_11_${page}.html`
}

// init方法
async function getAllImgUrl(){
    // 一共需要抓取1-49个地址
    let total = 49
    const allUrls = []
    for(let i=1; i<=total; i++){
        console.log(`开始抓取第${i}个网页`)
        let html = await getHtml(urlByPage(i))
        let $ = cheerio.load(html)
        // 找到页面上的所有图片元素
        let scripts = $('.pic2 script')

        for(let j=0; j<scripts.length; j++){
            let item = (scripts[j])
            let content = $(item).html()
            let res = /document\.getElementById\(\'.+?\'\)\.src\s=\s\"(.+?)\"/gs
            let img = res.exec(content)
            let title = $(item).prev().find('a').attr('title')
            let url = img[1]
            if(/^\/\//.test(url)){
                url = "http:" + url
            }
            if(img){
                let num = (i-1) * scripts.length + (j+1)
                //console.log(`正在下载第${num}张图片`)
                let size = await getImgSizeAndDownload(url)
                //console.log(`下载完毕第${num}张图片`)
                console.log(`${(num/(total*scripts.length)).toFixed(2)*100}%`)
                allUrls.push({
                    size,
                    url,
                    title
                })
            }
        }
    }
    // 存储分析数据
    fs.writeFile('./data.json', JSON.stringify(allUrls, function(key, value){
        if(!value){
            let reg = /<\s*\/?\s*\w+\s*>/ig
            return value.replace(reg,'')
        }else{
            return value
        }
    },4), function(err){
        if(!err){
            console.log('文件分析完毕')
        }
    })
    
}

// 获取网页内容
async function getHtml(url){
    let res = await axios.get(url,{
        responseType: 'stream'
    })
    return new Promise((resolve) => {
        const chunks = []
        res.data.on('data',chunk => {
            chunks.push(chunk)
        })
        res.data.on('end', () => {
            let buffer = Buffer.concat(chunks)
            let str = iconv.decode(buffer, 'GBK')
            resolve(str)
        })
    })
}

// 存储图片并返回图片大小
async function getImgSizeAndDownload(url){
    try{
        let res = await axios.get(url,{
            responseType: 'stream',
            "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36"
        })
        return new Promise((resolve) => {
            const chunks = []
            let pathUrl = path.join('images',path.basename(url))
            const ws = fs.createWriteStream(pathUrl)
            res.data.pipe(ws)
            res.data.on('data',chunk => {
                chunks.push(chunk)
            })
            res.data.on('end', () => {
                let buffer = Buffer.concat(chunks)
                resolve(buffer.length)
            })
        })
    }catch(e){
        console.log(e)
    }
}
getAllImgUrl()
